#!/usr/bin/env python

'''
to run from the command line:
pyytdl.py <search term(s)>

for example,
pyytdl.py 'mst3k manos'
'''

class Video(object):
    '''
    currently just a dummy struct-like class
    might add to this some day...
    '''
    pass


class VideoList(list):
    ''' '
>>> vl = VideoList(dev_id='Rand0mS7tr1ng',tag='"mst3k manos"',test=True)
>>> vl.url
'http://www.youtube.com/api2_rest?per_page=100&dev_id=Rand0mS7tr1ng&tag=%22mst3k%20manos%22&method=youtube.videos.list_by_tag&page=1'

an example response from youtube.com:
>>> vl.xml = r"""<?xml version="1.0" encoding="utf-8"?>
... <ut_response status="ok"><video_list><video><author>hangman86</author><id>rQwQEsOk2_U</id><title>MST3k - Manos: The Hands of Fate (Part 1/10)</title><length_seconds>576</length_seconds><rating_avg>4.94</rating_avg><rating_count>79</rating_count><description>Manos vs. Monster 
... 
... Which is more painful? Manos: The Hands of Fate or Monster a Go-Go.
... 
... I think the Eagles said it best about "Hotel Manos" when they sang:
... 
...  "You can check out anytime you like, but you can never be sure that you won't run into a deformed, stuttering, pervert whose boss is the leader of a satanic cult that kidnaps women and just might have a thing for your daughter as well! 
...    
... Note: I stole that Eagles reference from a comic named Doug Benson who was making fun of the movie "Hotel Rawanda."</description><view_count>12676</view_count><upload_time>1152583683</upload_time><comment_count>None</comment_count><tags>MST3k joel mystery science theater 3000 manos the hands of fate crow servo</tags><url>http://www.youtube.com/?v=rQwQEsOk2_U</url><thumbnail_url>http://sjl-static11.sjl.youtube.com/vi/rQwQEsOk2_U/2.jpg</thumbnail_url></video><video><author>Turnback</author><id>fKqh4bQWKUw</id><title>MST3K- Manos the hands of fate 2/11</title><length_seconds>592</length_seconds><rating_avg>5.00</rating_avg><rating_count>36</rating_count><description>Widely regarded as the most pungent piece of  ever hurled against the big screen, Manos-The Hands of Fate has long endured in legendary fashion as the holy grail of bad film-making.
... 
... Written, produced, directed by, and starring Hal Warren, a Texas fertilizer salesman who made a bet that he could he create a successful horror film with only $1000, Manos was filmed in one month with one camera capable of only 30 second shots and no sound. All voices were later dubbed in. 
... 
... At it's much trumpeted 1966 world premiere in El Paso, the cast quietly snuck out of the theater when the audience began laughing uncontrollably.  Within a year of it's release three of it's cast members committed suicide. Manos, however, lives on.  Enjoy.</description><view_count>6356</view_count><upload_time>1154360886</upload_time><comment_count>None</comment_count><tags>MST3K hired crow servo Joel Manos</tags><url>http://www.youtube.com/?v=fKqh4bQWKUw</url><thumbnail_url>http://sjl-static10.sjl.youtube.com/vi/fKqh4bQWKUw/2.jpg</thumbnail_url></video><video><title>MST3k - Manos: The Hands of Fate (Part 2/10)</title></video><video><title>MST3k - Manos: The Hands of Fate (Part 3/10)</title></video></video_list></ut_response>
... """
>>> #"
>>> vl._parse()
>>> vl[0].title
u'MST3k - Manos: The Hands of Fate (Part 1/10)'

errors from the server should be raised as an exception
>>> vl.xml = r'<?xml version="1.0" encoding="utf-8"?><ut_response status="fail"><error><code>8</code><description>Bad or unknown dev_id specified.</description></error></ut_response>'
>>> vl._parse() #doctest: +ELLIPSIS
Traceback (most recent call last):
UserWarning: Server Error: ...

internal functions -- users should not need to call these explicitly
>>> html = r"""<script type="text/javascript">
...		// <![CDATA[
...		var fo = new SWFObject("/player2.swf?hl=en&video_id=zFAazJohOo8&l=346&t=OEgsToPDskJeFlPB2hV20J4NP3CTAJ8x", "movie_player", "450", "370", 7, "#FFFFFF");
...             var swfArgs = {"video_id": "hRZzyM-xKaA", "l": 600, "sk": "dLEU-_-yp9U91Iwq_p1PiwC", "t": "OEgsToPDskKmugcBGyRj-acMdxNuVoKf", "hl": "en", "plid": "AARFxkD3e1DbNeOQAAAAAAAAAAQAAA"}:',)
...		fo.write("playerDiv");
...		// ]]>
...	</script>"""
>>> #"
>>> vl._parseID(html)
'hRZzyM-xKaA&l=600&t=OEgsToPDskKmugcBGyRj-acMdxNuVoKf'

the old format would have returned
'EyVsCfQCVg0&l=26&t=OEgsToPDskJkVX5HgKG2rVsvLjty-qLy'

the even older format would have returned
'zFAazJohOo8&l=346&t=OEgsToPDskJeFlPB2hV20J4NP3CTAJ8x'

need to be really careful to properly parse titles so that matching parts may
 be combined
>>> vl._parseTitle('MST3k - Manos: The Hands of Fate (Part 1/10)')
['MST', 'k - Manos: The Hands of Fate (Part ', '/', ')', 3, 1, 10]
>>> vl._parseTitle('manos the hands of fate part 1')
['manos the hands of fate part ', 1]
>>> vl._parseTitle('manos the hands of fate Part One (out of ten)')
['manos the hands of fate Part ', ' (out of ', ')', 1, 10]
>>> vl._parseTitle('manos 1 of 11')
['manos ', ' of ', 1, 11]
>>> vl._parseTitle('1/11 of Manos: The Hands of Fate')
['', '/', ' of Manos: The Hands of Fate', 1, 11]

._pickList is a key internal function that decides which of the (possibly) many
 videos to download
>>> vl._pickList()
[0, 2, 3]

' '''
    def __init__(self,tag=None,dev_id='',page=1,per_page=100,test=False):
        '''
        dev_id is the id given you by YouTube.
        '''
        import urllib
        assert tag
        params = {'method': 'youtube.videos.list_by_tag',
                  'dev_id': dev_id,
                  'tag': tag,
                  'page': page,
                  'per_page': per_page}
        p = urllib.urlencode(params).replace('+','%20') # urlencode makes
        # ' '->'+' for post requests
        self.url = r'http://www.youtube.com/api2_rest?'+p
        if not test:
            self._downloadList()
            self._parse()
        
    def _downloadList(self):
        import urllib
        f = urllib.urlopen(self.url)
        #self.xml = f.read()
        ls = f.readlines()
        self.xml = ''.join(ls)

    def _parse(self):
        from xml.dom.minidom import parseString
        self.dom = parseString(self.xml)
        assert self.dom.documentElement.tagName == 'ut_response'
        status = self.dom.documentElement.attributes['status'].nodeValue
        if status == 'fail': raise UserWarning,'Server Error: '+self.xml
        else: assert status == 'ok'
        for vx in self.dom.getElementsByTagName('video'):
            v = Video()
            # iterate over tags in video and assign attributes
            for n in vx.childNodes:
                assert len(n.childNodes) == 1
                assert n.childNodes[0].nodeType == n.TEXT_NODE
                v.__setattr__(n.nodeName,n.childNodes[0].nodeValue)
            self.append(v)

    def _parseTitle(self,text):
        import re
        # replace word numbers with digits
        nws = ['zero','one','two','three','four','five','six','seven','eight',
               'nine','ten','eleven','twelve','thirteen','fourteen','fifteen',
               'sixteen','seventeen','twenty']
        for nw in nws:
            n = str(nws.index(nw))
            text = re.sub(r'(?i)\b'+nw+r'\b',n,text)
        p = re.compile(r'(\D*)(\d+)')
        pEnd = re.compile(r'(\D*\d+)*(.*)')
        end = pEnd.search(text).groups()[1]
        ms = p.findall(text)
        l = []
        for m in ms: l.append(m[0])      # first put the strings in
        if end: l.append(end)
        for m in ms: l.append(int(m[1])) #  then put in the integers
        return l

    def _parseID(self,text):
        import re
        # old format
        #p = re.compile(r'SWFObject\("\/player2.swf\?.*video_id=(\S+)\"')
        #m = p.search(text)
        #if m: return m.groups()[0]
        #else: raise UserWarning, 'Video ID not found'
        # new format
        #p = re.compile(r"swfArgs = {hl:(.+),video_id:'(.+)',l:'(.+)',t:'(.+)',.*;")
        # newer format
        p = re.compile(r'swfArgs = {(.*?)"video_id": "(.+)",(.*?)"l": (.+?),(.*?)"t": "(.+?)",')
        m = p.search(text)
        if m:
            video_id,l,t = m.groups()[1:6:2]
            return video_id+'&l='+l+'&t='+t
        else:
            raise UserWarning, 'Video ID not found'
        
    def _pickList(self):
        ts = titles = []
        for v in self:
            ts.append(self._parseTitle(v.title))
        if not ts: raise UserWarning, 'No hits on search.'
        # build list of groups
        gs = self._groupTitles(ts)
        # pick the best group
        best = gs[0]
        for g in gs[1:]:
            # tests for quality:
            # 1. is there a list of integers?
            # 2. is the list free of holes? (such as 1,2,4,5,...)
            # 3. length of the list match the listed number? (eg, 3 of 6)
            #### 4. better resolution/framerate/compression?
            if self._listExists(g) and not self._listExists(best): best = g
            if self._noHoles(g) and not self._noHoles(best): best = g
            if self._goodLength(g) and not self._goodLength(best): best = g
        # pick out indices for the best group
        try: i = [isinstance(x,list) for x in best].index(True)
        except ValueError: return ts.index(best)
        l = best[i]
        l.sort()
        bi = []
        for j in l:
            best[i] = j
            bi.append(ts.index(best))   
        return bi
        #return range(len(self))

    def _listExists(self,g):
        try:
            i = [isinstance(x,list) for x in g].index(True)
            l = g[i]
            return [isinstance(x,int) for x in l].count(True) == len(l)
        except ValueError: 
            return False

    def _noHoles(self,g):
        try:
            l = g[[isinstance(x,list) for x in g].index(True)]
            for i in range(1,len(l)): assert i in l
            return True
        except (ValueError,AssertionError): return False

    def _goodLength(self,g):
        try:
            i = [isinstance(x,list) for x in g].index(True)
            l = g[i]
            return isinstance(g[i+1],int) and g[i+1] == len(l)
        except (ValueError,IndexError):
            # if there are two 1s in a row
            return [x==1 and y==1 for x,y in zip(g[:-1],g[1:])].count(True) > 1
            

    def _groupTitles(self,titles):
        import copy
        # need to use a copy of ts so refs don't screw it up
        ts = copy.deepcopy(titles)
        gs = groups = []
        for t in ts:
            inserted = False
            for i in range(len(gs)):
                if len(gs[i]) != len(t):
                    continue
                diff = [gItem == tItem for gItem,tItem in zip(gs[i],t)]
                ndiff = diff.count(False)
                if ndiff == 0:
                    #raise UserWarning, 'repeated title'
                    continue
                j = diff.index(False)
                if ndiff == 1:
                    if not isinstance(gs[i][j],list):
                        gs[i][j] = [gs[i][j],]
                    if t[j] in gs[i][j]:
                        #raise UserWarning,'repeated title'
                        continue
                    else:
                        gs[i][j].append(t[j])
                        inserted = True
                elif ndiff == 2:
                    if not isinstance(gs[i][j],list):
                        j2 = j
                        j = diff.index(False,j+1)
                        if not isinstance(gs[i][j],list):
                            gs.append(t)
                            inserted = True
                            continue
                    else: j2 = diff.index(False,j+1)
                    # j = index to list element of gs[i]
                    # j2 = index to the other difference
                    if t[j] in gs[i][j]:
                        t[j2] = [gs[i][j2], t[j2]]
                        gs.append(t)
                        inserted = True
                else:
                    gs.append(t)
                    inserted = True
            if not inserted: gs.append(t)
        return gs

    def downloadVideo(self,video,filename):
        import urllib
        f = urllib.urlopen(video.url)
        h = f.read()
        f.close()
        vId = self._parseID(h)
        urllib.urlretrieve(r"http://www.youtube.com/get_video?video_id="+vId,
                           filename)
    def downloadVideos(self,indices,test=False):
        import os
        if not isinstance(indices,list): indices = [indices,]
        for i in indices:
            filename = self[i].title.replace('/','_')+'.flv'
            if os.path.exists(filename):
                print 'skipping (exists)', filename
            else:
                print 'downloading', filename
                if not test:
                    self.downloadVideo(self[i],filename)

    def downloadPickedVideos(self,test=False):
        self.downloadVideos(self._pickList(),test)

if __name__ == '__main__':
    import sys
    import os
    dev_id = os.environ['YOUTUBE_DEV_ID']
    tag = ' '.join(sys.argv[1:]).strip('"').strip("'")
    vl = VideoList(dev_id=dev_id,tag=tag)
    #vl.downloadPickedVideos(test=True)
    vl.downloadPickedVideos()
    #vl.downloadVideos(range(0,17),test=True)
    #vl.downloadVideos(range(0,17),test=False)
    #import pdb;pdb.set_trace()
    
